from config.Config import BASE_DIR
from cio.jsonl import JsonLReader, JsonLWriter
import pickle
import random
from tqdm import tqdm

data = JsonLReader(BASE_DIR / 'resource' / 'data' / 'consum' / 'split_context.train.jsonl').yield_read()

count = 0
total = 0
for each in tqdm(data):
    context = each['context']
    method = each['method']
    summary = each['summary']
    for word in summary:
        if word in context and word not in method:
            count = count + 1
            break
    total += 1

print(total, count)
